{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "73e08a01-c5a1-4bfe-b922-1659b9935eb7",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-02-20T01:17:09.022992Z",
     "iopub.status.busy": "2025-02-20T01:17:09.022678Z",
     "iopub.status.idle": "2025-02-20T01:17:09.040934Z",
     "shell.execute_reply": "2025-02-20T01:17:09.040439Z",
     "shell.execute_reply.started": "2025-02-20T01:17:09.022974Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "from jinja2 import Template\n",
    "data = pd.read_json('./super_long_str_hecheng472.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "60c5a161-af94-41ff-bf21-33a73d26ecf2",
   "metadata": {
    "ExecutionIndicator": {
     "show": false
    },
    "execution": {
     "iopub.execute_input": "2025-02-19T07:41:17.412308Z",
     "iopub.status.busy": "2025-02-19T07:41:17.411956Z",
     "iopub.status.idle": "2025-02-19T07:41:17.415857Z",
     "shell.execute_reply": "2025-02-19T07:41:17.415394Z",
     "shell.execute_reply.started": "2025-02-19T07:41:17.412286Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# sys = '''\n",
    "# 你是一个有帮助的助手，可以访问以下功能函数：\n",
    "# {{api_list}}\n",
    "\n",
    "# 1、请根据用户的对话内容判断是否需要调用功能函数列表中函数以及选择恰当调用的时机。\n",
    "# 2、要使用这些功能，请回复：  {\"name\": \"function_name\", \"arguments\": {\"arg_1\": \"value_1\", \"arg_1\": \"value_1\", ...}}，name是调用的函数的名称，arguments是函数所需的参数。\n",
    "# 3、你必须处理的边缘情况： - 如果没有与用户请求匹配的功能，你将礼貌地回应说你无法帮助。\"\n",
    "# '''\n",
    "# template_sys=Template(sys)\n",
    "# system=[]\n",
    "# for index,row in data.iterrows():\n",
    "#     input_string = row['system']\n",
    "#     start_str = \"-\"\n",
    "#     start_idx = input_string.find(start_str) + len(start_str)\n",
    "\n",
    "#     # 查找结束点\n",
    "#     end_str = \"。要使用这些功能\"\n",
    "#     end_idx = input_string.find(end_str, start_idx)\n",
    "\n",
    "#     # 提取内容\n",
    "#     extracted_content = input_string[start_idx:end_idx].strip()\n",
    "#     system.append(template_sys.render(api_list=json.loads(extracted_content)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b71529ec-7650-461c-8382-d0dabfd56051",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-02-20T01:17:11.666771Z",
     "iopub.status.busy": "2025-02-20T01:17:11.666420Z",
     "iopub.status.idle": "2025-02-20T01:17:11.669765Z",
     "shell.execute_reply": "2025-02-20T01:17:11.669293Z",
     "shell.execute_reply.started": "2025-02-20T01:17:11.666751Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "data_list = list(data['conversations'])\n",
    "data_list_sys = list(data['system'])\n",
    "# data_list_tools = list(data['tools'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b4dcecb6-b1a7-427f-808f-84ced5725e98",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-02-20T01:17:14.142192Z",
     "iopub.status.busy": "2025-02-20T01:17:14.141874Z",
     "iopub.status.idle": "2025-02-20T01:17:14.159327Z",
     "shell.execute_reply": "2025-02-20T01:17:14.158873Z",
     "shell.execute_reply.started": "2025-02-20T01:17:14.142173Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 472/472 [00:00<00:00, 110431.83it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import json\n",
    "error_l = []\n",
    "c = 0\n",
    "for i in tqdm(data_list):\n",
    "    try:\n",
    "        json.loads(i)\n",
    "    except ValueError:\n",
    "        error_l.append(c)\n",
    "    c+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "e3e724c7-c0b5-4236-920f-308501bd605c",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-02-14T07:30:12.740095Z",
     "iopub.status.busy": "2025-02-14T07:30:12.739794Z",
     "iopub.status.idle": "2025-02-14T07:30:12.743202Z",
     "shell.execute_reply": "2025-02-14T07:30:12.742747Z",
     "shell.execute_reply.started": "2025-02-14T07:30:12.740075Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "indices_to_remove = error_l\n",
    "\n",
    "# 按照索引从大到小排序，这样在删除时不会影响前面元素的位置\n",
    "for index in sorted(indices_to_remove, reverse=True):\n",
    "    if 0 <= index < len(data_list): # 确保索引在列表范围内\n",
    "        del data_list[index]\n",
    "        del data_list_sys[index]\n",
    "        del data_list_tools[index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7af4ebec-2296-407d-aaa1-f145a8bc3278",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-02-20T01:17:28.998418Z",
     "iopub.status.busy": "2025-02-20T01:17:28.998085Z",
     "iopub.status.idle": "2025-02-20T01:17:29.006719Z",
     "shell.execute_reply": "2025-02-20T01:17:29.006275Z",
     "shell.execute_reply.started": "2025-02-20T01:17:28.998399Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 472/472 [00:00<00:00, 108039.26it/s]\n"
     ]
    }
   ],
   "source": [
    "for i in tqdm(list(data_list)):\n",
    "    json.loads(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "95f9578b-75a4-4b94-b7ed-815733f9dbb0",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-02-20T01:18:30.946249Z",
     "iopub.status.busy": "2025-02-20T01:18:30.945922Z",
     "iopub.status.idle": "2025-02-20T01:18:31.074764Z",
     "shell.execute_reply": "2025-02-20T01:18:31.074246Z",
     "shell.execute_reply.started": "2025-02-20T01:18:30.946229Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = pd.DataFrame()\n",
    "for conver,sys in zip(data_list,data_list_sys):\n",
    "    react = json.loads(conver)\n",
    "    new_row = pd.DataFrame([{'conversations':react,'system':sys}])\n",
    "    df = pd.concat([df,new_row],ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "82dd0451-bf0b-4975-8cf5-b3742e37b0db",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-02-20T01:18:56.576083Z",
     "iopub.status.busy": "2025-02-20T01:18:56.575753Z",
     "iopub.status.idle": "2025-02-20T01:18:56.622052Z",
     "shell.execute_reply": "2025-02-20T01:18:56.621559Z",
     "shell.execute_reply.started": "2025-02-20T01:18:56.576063Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.to_json('new_super_long_str_hecheng472.json',orient='records',force_ascii=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04c21ddc-56af-46cd-af75-0232bf8008bb",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import json\n",
    "s_h = pd.read_json('./new_super_long_str_hecheng472.json')\n",
    "data_list,data_list_sys=list(s_h['conversations']), list(s_h['system'])\n",
    "conver,sys = list(s_h['conversations']), list(s_h['system'])\n",
    "i = 0\n",
    "error_l=[]\n",
    "for con,system in zip(conver,sys):\n",
    "    \n",
    "    for entry in con:\n",
    "        if entry['from']=='function_call':\n",
    "            # print(entry['name'])\n",
    "            # print(entry['value'])\n",
    "            try:\n",
    "                test=json.loads(entry['value'])\n",
    "                print(test['name'])\n",
    "            except KeyError:\n",
    "                error_l.append(i)\n",
    "    i+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d554a9a-6c66-4228-8ef3-90cc7285c5e4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "81b95cb6-35bb-42db-9744-525d227d7e29",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-02-20T01:46:26.379698Z",
     "iopub.status.busy": "2025-02-20T01:46:26.379336Z",
     "iopub.status.idle": "2025-02-20T01:46:26.419272Z",
     "shell.execute_reply": "2025-02-20T01:46:26.418774Z",
     "shell.execute_reply.started": "2025-02-20T01:46:26.379677Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "s_h.drop(error_l,axis=0).reset_index().drop(columns=['index'],axis=1).to_json('new_super_long_str_hecheng470.json',orient='records',force_ascii=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "9399a2ed-ebcc-4000-b11d-60c6b1812b9f",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-02-20T01:46:45.742575Z",
     "iopub.status.busy": "2025-02-20T01:46:45.742230Z",
     "iopub.status.idle": "2025-02-20T01:46:45.756885Z",
     "shell.execute_reply": "2025-02-20T01:46:45.756310Z",
     "shell.execute_reply.started": "2025-02-20T01:46:45.742554Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 's_sh' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[34], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43ms_sh\u001b[49m\u001b[38;5;241m.\u001b[39mto_json(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m./super_hecheng.json\u001b[39m\u001b[38;5;124m'\u001b[39m,orient\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mrecords\u001b[39m\u001b[38;5;124m'\u001b[39m,force_ascii\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n",
      "\u001b[0;31mNameError\u001b[0m: name 's_sh' is not defined"
     ]
    }
   ],
   "source": [
    "s_sh.to_json('./super_hecheng.json',orient='records',force_ascii=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
